# -*- coding: utf-8 -*-
from scrapy_redis.spiders import RedisSpider
import json
from scrapy.http import Request
from bs4 import BeautifulSoup
from ..items import *
import time


class MoviesSpider(RedisSpider):
    '''
    该爬虫是用来下载所有电影的详细信息
    '''
    name = 'movies'
    allowed_domains = ['www.douban.com']

    # 定义需要爬取的电影标签和其对应的数量
    def parse(self, response):
        temp = {
            '豆瓣高分': 500,  # 0-240, 240-500
            '热门': 400,
            '经典': 500,
            '冷门佳片': 500,
        }

        # 把对应标签的所有电影整合
        for key, page in temp.items():
            i = 0
            while i < page:
                url = 'https://movie.douban.com/j/search_subjects?type=movie&tag=' + key + '&sort=recommend&page_limit=20&page_start=' + str(
                    i)
                yield Request(
                    url=url,
                    callback=self.get_movie_url,
                    dont_filter=True,
                )

                i += 20

    # 处理电影基本信息
    def get_movie_url(self, response):
        '''

        :param response: json格式；其包含电影的基本信息
        :return:
        '''

        content = json.loads(response.text)
        items = content['subjects']

        for item in items:
            mid = item['id']

            # 处理电影详细信息调度器
            yield Request(
                url='https://movie.douban.com/j/subject_abstract?subject_id=' + mid,
                callback=self.get_movie_json,
                meta={'img_url': item['cover']},
                dont_filter=True,
            )

    # 处理电影详细信息
    def get_movie_json(self, response):
        content = json.loads(response.text)
        info = content['subject']
        info['img_url'] = response.meta['img_url']

        mid = info['id']
        yield Request(
            url='https://movie.douban.com/subject/' + mid + '/',
            callback=self.get_movie_info,
            meta={'info': info},
            dont_filter=True,
        )

    # 处理电影详细信息进管道
    def get_movie_info(self, response):
        content = response.text
        soup = BeautifulSoup(content)

        intro = soup.select_one(".related-info span[property]").text
        info = response.meta['info']

        movie = CutMovieItem()
        movie['mid'] = info['id']
        movie['title'] = info['title']
        movie['rate'] = info['rate']
        movie['directors'] = '-'.join(info['directors'])
        movie['actors'] = '-'.join(info['actors'])
        movie['types'] = '-'.join(info['types'])
        movie['region'] = info['region']
        movie['release_year'] = info['release_year']
        movie['img_url'] = info['img_url']
        movie['intro'] = intro

        yield movie
